#!./common/bats/bin/bats
# -*-sh-*-

load ./common/test_helper_functions || exit 1
source ./common/functions || exit 1

if [ -s ./common/TEST_ENV ];then
   . ./common/TEST_ENV
else
    ERROR "TEST_ENV does not exist"
fi

CMD_TIMEOUT="15:00"
TEST_EXE="./src/mini_dft"
TEST_MAX_COMPUTES=4
if [ $LMOD_FAMILY_MPI == "impi" ];then
    TEST_NUM_RANKS=2
else
    TEST_NUM_RANKS=4
fi
TEST_NUM_THREADS=2
TEST_VERIFY=1
TEST_PERFLOG=./log.miniDFT

check_rms
rm=$RESOURCE_MANAGER

# init run options
tdir=./test
sample="Si_333"
ifile=$sample.in
ofile=$sample.out
rfile=QE_$sample.out.ref
rtext="!    total energy"

NODES=$((NUM_COMPUTES < TEST_MAX_COMPUTES ? NUM_COMPUTES : TEST_MAX_COMPUTES))
TASKS=$((NODES*TEST_NUM_RANKS))
ARGS="-in $ifile"
EXE=$TEST_EXE.$LMOD_FAMILY_COMPILER.$LMOD_FAMILY_MPI

# set global env settings
export OMP_NUM_THREADS=$TEST_NUM_THREADS
if [ $LMOD_FAMILY_MPI == "mvapich2" ];then
    export MV2_ENABLE_AFFINITY=0
    export IPATH_NO_CPUAFFINITY=1
    # temporary workaround pending investigation on issue related to IB registration cache
    if [ $LMOD_FAMILY_COMPILER == "gnu" ];then
        export MV2_VBUF_TOTAL_SIZE=8000000
        export MV2_IBA_EAGER_THRESHOLD=8000000
    fi
fi

@test "[Apps/miniDFT] run miniDFT on multi nodes under resource manager ($rm/$LMOD_FAMILY_COMPILER/$LMOD_FAMILY_MPI)" {
    rm -f $tdir/$ofile $tdir/job.*.out

    if [ ! -x $EXE ];then
        flunk "missing $EXE"
    fi

    if [ ! -s $tdir/$ifile ];then
        flunk "missing $ifile"
    fi

    pushd $tdir >& /dev/null
    run_mpi_binary -t $CMD_TIMEOUT -o $ofile ../$EXE "$ARGS" $NODES $TASKS
    assert_success
    popd >& /dev/null
}

if [ "$TEST_VERIFY"=="1" ];then
@test "[Apps/miniDFT] validate miniDFT multi node output ($rm/$LMOD_FAMILY_COMPILER/$LMOD_FAMILY_MPI)" {
    diff <(grep "$rtext" $tdir/$rfile) <(grep "$rtext" $tdir/$ofile)
    assert_success
}
fi

@test "[Apps/miniDFT] log miniDFT multi node results ($rm/$LMOD_FAMILY_COMPILER/$LMOD_FAMILY_MPI)" {
    run_tag=miniDFT.${sample}.P${TASKS}
    wfile=$run_tag.$LMOD_FAMILY_COMPILER.$LMOD_FAMILY_MPI.out
    mv $tdir/$ofile $tdir/$wfile || exit 1

    str="Benchmark_Time"
    ref_time=`grep $str $tdir/ref/$wfile | sed 's/.*CPU \(.*\)s WALL.*/\1/'`
    cur_time=`grep $str $tdir/$wfile     | sed 's/.*CPU \(.*\)s WALL.*/\1/'`

    if [ -z $ref_time ];then
        ref_time="?"
    fi

    if [ -z $cur_time ];then
        cur_time="?"
    fi

    fmt="%-24s %-10s %-10s %16s %16s\n"
    printf "$fmt" $run_tag $LMOD_FAMILY_COMPILER $LMOD_FAMILY_MPI $ref_time $cur_time >>$TEST_PERFLOG
}
